import requests
from bs4 import BeautifulSoup
import time
import re

base_url='http://www.nv43.com'

headers = {'Accept': 'text/html, application/xhtml+xml, image/jxr, */*',
               'Accept - Encoding':'gzip, deflate',
               'Accept-Language':'zh-Hans-CN, zh-Hans; q=0.5',
               'Connection':'Keep-Alive',
               'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063'}
imgurl_list=[]


for i in range(1,7):
    url='http://www.nv43.com/tuku/xieedongtaitu/list_'+str(i)+'.html'

    r = requests.get(url)
    content=r.content
    soup = BeautifulSoup(content,"lxml")

    pa=soup.find('ul',attrs={'class':'big_img_ul'})

    imgset_list=pa.find_all('a')
    time.sleep(0.1)

    for imgset in imgset_list:
        imgset_url=base_url+imgset['href']

        rr = requests.get(imgset_url)
        ccontent=rr.content
        ssoup = BeautifulSoup(ccontent,"lxml")

        ndiv=ssoup.find('div',attrs={'class':'navpage'})
        pagenum=int(re.sub("\D", "", ndiv.find('span').text))

        time.sleep(0.1)

        for pn in range(2,pagenum+1):
            img_detail_url=imgset_url[:-5]+'_'+str(pn)+'.html'

            rrr = requests.get(img_detail_url)
            cccontent=rrr.content
            sssoup = BeautifulSoup(cccontent,"lxml")


            ddiv=sssoup.find('div',attrs={'class':'news_body'})
            imgs=ddiv.find_all('img')
            for img in imgs:
                imgurl_list.append(img['src'])
            time.sleep(0.1)

print('strat write to disk...')
with open('nv43gifurl.txt',mode='a+') as file:
    for imgurl in imgurl_list:
        file.write(imgurl+'\n')


        